volatile int ref_count;
ThreadClosure *thread_closure;
- SoupURI *uri;
+ GPtrArray *mirrorlist; /* list of base URIs */
+ char *filename; /* relative name to fetch or NULL */
+ guint mirrorlist_idx;
OstreeFetcherState state;
g_clear_pointer (&pending->thread_closure, thread_closure_unref);
- soup_uri_free (pending->uri);
+ g_clear_pointer (&pending->mirrorlist, g_ptr_array_unref);
+ g_free (pending->filename);
g_clear_object (&pending->request);
g_clear_object (&pending->request_body);
g_free (pending->out_tmpfile);
}
}
+static void
+create_pending_soup_request (OstreeFetcherPendingURI *pending,
+ GError **error)
+{
+ g_autofree char *uristr = NULL;
+ SoupURI *next_mirror = NULL;
+ SoupURI *uri = NULL;
+
+ g_assert (pending->mirrorlist);
+ g_assert (pending->mirrorlist_idx < pending->mirrorlist->len);
+
+ next_mirror = g_ptr_array_index (pending->mirrorlist,
+ pending->mirrorlist_idx);
+ uristr = g_build_filename (soup_uri_get_path (next_mirror),
+ pending->filename /* may be NULL */, NULL);
+ uri = soup_uri_copy (next_mirror);
+ soup_uri_set_path (uri, uristr);
+
+ g_clear_object (&pending->request);
+
+ pending->request = soup_session_request_uri (pending->thread_closure->session,
+ uri, error);
+ soup_uri_free (uri);
+}
+
static void
session_thread_request_uri (ThreadClosure *thread_closure,
gpointer data)
pending = g_task_get_task_data (task);
cancellable = g_task_get_cancellable (task);
- pending->request = soup_session_request_uri (thread_closure->session,
- pending->uri,
- &local_error);
-
+ create_pending_soup_request (pending, &local_error);
if (local_error != NULL)
{
g_task_return_error (task, local_error);
}
else
{
- g_autofree char *uristring = soup_uri_to_string (pending->uri, FALSE);
+ g_autofree char *uristring
+ = soup_uri_to_string (soup_request_get_uri (pending->request), FALSE);
g_autofree char *tmpfile = NULL;
struct stat stbuf;
gboolean exists;
SOUP_SESSION_IDLE_TIMEOUT, 60,
NULL);
+ /* XXX: Now that we have mirrorlist support, we could make this even smarter
+ * by spreading requests across mirrors. */
g_object_get (closure->session, "max-conns-per-host", &max_conns, NULL);
if (max_conns < 8)
{
if (bytes_read > pending->max_size ||
(bytes_read + pending->current_size) > pending->max_size)
{
- g_autofree char *uristr = soup_uri_to_string (pending->uri, FALSE);
+ g_autofree char *uristr =
+ soup_uri_to_string (soup_request_get_uri (pending->request), FALSE);
local_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED,
"URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes",
uristr, pending->max_size);
}
else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
{
- GIOErrorEnum code;
- switch (msg->status_code)
+ /* is there another mirror we can try? */
+ if (pending->mirrorlist_idx + 1 < pending->mirrorlist->len)
+ {
+ pending->mirrorlist_idx++;
+ create_pending_soup_request (pending, &local_error);
+ if (local_error != NULL)
+ goto out;
+
+ (void) g_input_stream_close (pending->request_body, NULL, NULL);
+ g_queue_insert_sorted (&pending->thread_closure->pending_queue,
+ g_object_ref (task), pending_task_compare,
+ NULL);
+ remove_pending_rerun_queue (pending);
+ }
+ else
{
- case 404:
- case 410:
- code = G_IO_ERROR_NOT_FOUND;
- break;
- default:
- code = G_IO_ERROR_FAILED;
+ GIOErrorEnum code;
+ switch (msg->status_code)
+ {
+ case 404:
+ case 410:
+ code = G_IO_ERROR_NOT_FOUND;
+ break;
+ default:
+ code = G_IO_ERROR_FAILED;
+ }
+
+ local_error = g_error_new (G_IO_ERROR, code,
+ "Server returned status %u: %s",
+ msg->status_code,
+ soup_status_get_phrase (msg->status_code));
+
+ if (pending->mirrorlist->len > 1)
+ g_prefix_error (&local_error,
+ "All %u mirrors failed. Last error was: ",
+ pending->mirrorlist->len);
}
- local_error = g_error_new (G_IO_ERROR, code,
- "Server returned status %u: %s",
- msg->status_code,
- soup_status_get_phrase (msg->status_code));
goto out;
}
}
}
static void
-ostree_fetcher_request_uri_internal (OstreeFetcher *self,
- SoupURI *uri,
- gboolean is_stream,
- guint64 max_size,
- int priority,
- GCancellable *cancellable,
- GAsyncReadyCallback callback,
- gpointer user_data,
- gpointer source_tag)
+ostree_fetcher_mirrored_request_internal (OstreeFetcher *self,
+ GPtrArray *mirrorlist,
+ const char *filename,
+ gboolean is_stream,
+ guint64 max_size,
+ int priority,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data,
+ gpointer source_tag)
{
g_autoptr(GTask) task = NULL;
OstreeFetcherPendingURI *pending;
g_return_if_fail (OSTREE_IS_FETCHER (self));
- g_return_if_fail (uri != NULL);
+ g_return_if_fail (mirrorlist != NULL);
+ g_return_if_fail (mirrorlist->len > 0);
/* SoupRequest is created in session thread. */
pending = g_new0 (OstreeFetcherPendingURI, 1);
pending->ref_count = 1;
pending->thread_closure = thread_closure_ref (self->thread_closure);
- pending->uri = soup_uri_copy (uri);
+ pending->mirrorlist = g_ptr_array_ref (mirrorlist);
+ pending->filename = g_strdup (filename);
pending->max_size = max_size;
pending->is_stream = is_stream;
}
void
-_ostree_fetcher_request_uri_with_partial_async (OstreeFetcher *self,
- SoupURI *uri,
- guint64 max_size,
- int priority,
- GCancellable *cancellable,
- GAsyncReadyCallback callback,
- gpointer user_data)
+_ostree_fetcher_mirrored_request_with_partial_async (OstreeFetcher *self,
+ GPtrArray *mirrorlist,
+ const char *filename,
+ guint64 max_size,
+ int priority,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data)
{
- ostree_fetcher_request_uri_internal (self, uri, FALSE, max_size, priority, cancellable,
- callback, user_data,
- _ostree_fetcher_request_uri_with_partial_async);
+ ostree_fetcher_mirrored_request_internal (self, mirrorlist, filename, FALSE,
+ max_size, priority, cancellable,
+ callback, user_data,
+ _ostree_fetcher_mirrored_request_with_partial_async);
}
char *
-_ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher *self,
- GAsyncResult *result,
- GError **error)
+_ostree_fetcher_mirrored_request_with_partial_finish (OstreeFetcher *self,
+ GAsyncResult *result,
+ GError **error)
{
g_return_val_if_fail (g_task_is_valid (result, self), NULL);
g_return_val_if_fail (g_async_result_is_tagged (result,
- _ostree_fetcher_request_uri_with_partial_async), NULL);
+ _ostree_fetcher_mirrored_request_with_partial_async), NULL);
return g_task_propagate_pointer (G_TASK (result), error);
}
static void
-ostree_fetcher_stream_uri_async (OstreeFetcher *self,
- SoupURI *uri,
- guint64 max_size,
- int priority,
- GCancellable *cancellable,
- GAsyncReadyCallback callback,
- gpointer user_data)
+ostree_fetcher_stream_mirrored_uri_async (OstreeFetcher *self,
+ GPtrArray *mirrorlist,
+ const char *filename,
+ guint64 max_size,
+ int priority,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data)
{
- ostree_fetcher_request_uri_internal (self, uri, TRUE, max_size, priority, cancellable,
- callback, user_data,
- ostree_fetcher_stream_uri_async);
+ ostree_fetcher_mirrored_request_internal (self, mirrorlist, filename, TRUE,
+ max_size, priority, cancellable,
+ callback, user_data,
+ ostree_fetcher_stream_mirrored_uri_async);
}
static GInputStream *
-ostree_fetcher_stream_uri_finish (OstreeFetcher *self,
- GAsyncResult *result,
- GError **error)
+ostree_fetcher_stream_mirrored_uri_finish (OstreeFetcher *self,
+ GAsyncResult *result,
+ GError **error)
{
g_return_val_if_fail (g_task_is_valid (result, self), NULL);
g_return_val_if_fail (g_async_result_is_tagged (result,
- ostree_fetcher_stream_uri_async), NULL);
+ ostree_fetcher_stream_mirrored_uri_async), NULL);
return g_task_propagate_pointer (G_TASK (result), error);
}
{
FetchUriSyncData *data = user_data;
- data->result_stream = ostree_fetcher_stream_uri_finish ((OstreeFetcher*)object,
- result, data->error);
+ data->result_stream = ostree_fetcher_stream_mirrored_uri_finish ((OstreeFetcher*)object,
+ result, data->error);
data->done = TRUE;
}
gboolean
-_ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
- SoupURI *uri,
- gboolean add_nul,
- gboolean allow_noent,
- GBytes **out_contents,
- guint64 max_size,
- GCancellable *cancellable,
- GError **error)
+_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
+ GPtrArray *mirrorlist,
+ const char *filename,
+ gboolean add_nul,
+ gboolean allow_noent,
+ GBytes **out_contents,
+ guint64 max_size,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
const guint8 nulchar = 0;
data.done = FALSE;
data.error = error;
- ostree_fetcher_stream_uri_async (fetcher, uri,
- max_size,
- OSTREE_FETCHER_DEFAULT_PRIORITY,
- cancellable,
+ ostree_fetcher_stream_mirrored_uri_async (fetcher, mirrorlist, filename, max_size,
+ OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
fetch_uri_sync_on_complete, &data);
while (!data.done)
g_main_context_iteration (mainctx, TRUE);
g_clear_object (&(data.result_stream));
return ret;
}
+
+/* Helper for callers who just want to fetch single one-off URIs */
+gboolean
+_ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
+ SoupURI *uri,
+ gboolean add_nul,
+ gboolean allow_noent,
+ GBytes **out_contents,
+ guint64 max_size,
+ GCancellable *cancellable,
+ GError **error)
+{
+ g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
+ g_ptr_array_add (mirrorlist, uri); /* no transfer */
+ return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL,
+ add_nul, allow_noent,
+ out_contents, max_size,
+ cancellable, error);
+}
char *remote_name;
OstreeRepoMode remote_mode;
OstreeFetcher *fetcher;
- SoupURI *base_uri;
- SoupURI *base_content_uri;
+ GPtrArray *meta_mirrorlist; /* List of base URIs for fetching metadata */
+ GPtrArray *content_mirrorlist; /* List of base URIs for fetching content */
OstreeRepo *remote_repo_local;
GMainContext *main_context;
guint recursion_depth;
} ScanObjectQueueData;
-static SoupURI *
-suburi_new (SoupURI *base,
- const char *first,
- ...) G_GNUC_NULL_TERMINATED;
-
static void queue_scan_one_metadata_object (OtPullData *pull_data,
const char *csum,
OstreeObjectType objtype,
GCancellable *cancellable,
GError **error);
-static SoupURI *
-suburi_new (SoupURI *base,
- const char *first,
- ...)
-{
- va_list args;
- GPtrArray *arg_array;
- const char *arg;
- char *subpath;
- SoupURI *ret;
-
- arg_array = g_ptr_array_new ();
- g_ptr_array_add (arg_array, (char*)soup_uri_get_path (base));
- g_ptr_array_add (arg_array, (char*)first);
-
- va_start (args, first);
-
- while ((arg = va_arg (args, const char *)) != NULL)
- g_ptr_array_add (arg_array, (char*)arg);
- g_ptr_array_add (arg_array, NULL);
-
- subpath = g_build_filenamev ((char**)arg_array->pdata);
- g_ptr_array_unref (arg_array);
-
- ret = soup_uri_copy (base);
- soup_uri_set_path (ret, subpath);
- g_free (subpath);
-
- va_end (args);
-
- return ret;
-}
-
static gboolean
update_progress (gpointer user_data)
{
} OstreeFetchUriSyncData;
static gboolean
-fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher,
- SoupURI *uri,
- char **out_contents,
- GCancellable *cancellable,
- GError **error)
+fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher,
+ GPtrArray *mirrorlist,
+ const char *filename,
+ char **out_contents,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
g_autoptr(GBytes) bytes = NULL;
g_autofree char *ret_contents = NULL;
gsize len;
- if (!_ostree_fetcher_request_uri_to_membuf (fetcher, uri, TRUE,
- FALSE, &bytes,
- OSTREE_MAX_METADATA_SIZE,
- cancellable, error))
+ if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist,
+ filename, TRUE, FALSE,
+ &bytes,
+ OSTREE_MAX_METADATA_SIZE,
+ cancellable, error))
goto out;
ret_contents = g_bytes_unref_to_data (bytes, &len);
return ret;
}
+static gboolean
+fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher,
+ SoupURI *uri,
+ char **out_contents,
+ GCancellable *cancellable,
+ GError **error)
+{
+ g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
+ g_ptr_array_add (mirrorlist, uri); /* no transfer */
+ return fetch_mirrored_uri_contents_utf8_sync (fetcher, mirrorlist,
+ NULL, out_contents,
+ cancellable, error);
+}
+
static gboolean
write_commitpartial_for (OtPullData *pull_data,
const char *checksum,
{
gboolean ret = FALSE;
g_autofree char *ret_contents = NULL;
- SoupURI *target_uri = NULL;
+ g_autofree char *filename = NULL;
- target_uri = suburi_new (pull_data->base_uri, "refs", "heads", ref, NULL);
+ filename = g_build_filename ("refs", "heads", ref, NULL);
- if (!fetch_uri_contents_utf8_sync (pull_data->fetcher, target_uri,
- &ret_contents, cancellable, error))
+ if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher,
+ pull_data->meta_mirrorlist,
+ filename, &ret_contents,
+ cancellable, error))
goto out;
g_strchomp (ret_contents);
ret = TRUE;
ot_transfer_out_value (out_contents, &ret_contents);
out:
- if (target_uri)
- soup_uri_free (target_uri);
return ret;
}
OstreeObjectType objtype;
gboolean free_fetch_data = TRUE;
- temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error);
+ temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
if (!temp_path)
goto out;
g_debug ("fetch of %s%s complete", checksum_obj,
fetch_data->is_detached_meta ? " (detached)" : "");
- temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error);
+ temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
if (!temp_path)
{
if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
- temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error);
+ temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
if (!temp_path)
goto out;
gboolean is_detached_meta,
gboolean object_is_stored)
{
- SoupURI *obj_uri = NULL;
+ g_autofree char *obj_subpath = NULL;
gboolean is_meta;
FetchObjectData *fetch_data;
- g_autofree char *objpath = NULL;
guint64 *expected_max_size_p;
guint64 expected_max_size;
+ GPtrArray *mirrorlist = NULL;
g_debug ("queuing fetch of %s.%s%s", checksum,
ostree_object_type_to_string (objtype),
{
char buf[_OSTREE_LOOSE_PATH_MAX];
_ostree_loose_path (buf, checksum, OSTREE_OBJECT_TYPE_COMMIT_META, pull_data->remote_mode);
- obj_uri = suburi_new (pull_data->base_uri, "objects", buf, NULL);
+ obj_subpath = g_build_filename ("objects", buf, NULL);
+ mirrorlist = pull_data->meta_mirrorlist;
}
else
{
- objpath = _ostree_get_relative_object_path (checksum, objtype, TRUE);
- obj_uri = suburi_new (pull_data->base_content_uri, objpath, NULL);
+ obj_subpath = _ostree_get_relative_object_path (checksum, objtype, TRUE);
+ mirrorlist = pull_data->content_mirrorlist;
}
is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
else
expected_max_size = 0;
- _ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, obj_uri,
- expected_max_size,
- is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
- : OSTREE_REPO_PULL_CONTENT_PRIORITY,
- pull_data->cancellable,
- is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
- soup_uri_free (obj_uri);
+ _ostree_fetcher_mirrored_request_with_partial_async (pull_data->fetcher, mirrorlist,
+ obj_subpath, expected_max_size,
+ is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
+ : OSTREE_REPO_PULL_CONTENT_PRIORITY,
+ pull_data->cancellable,
+ is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
}
static gboolean
gboolean ret = FALSE;
g_autofree char *contents = NULL;
GKeyFile *ret_keyfile = NULL;
- SoupURI *target_uri = NULL;
- target_uri = suburi_new (pull_data->base_uri, "config", NULL);
-
- if (!fetch_uri_contents_utf8_sync (pull_data->fetcher, target_uri, &contents,
- cancellable, error))
+ if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher,
+ pull_data->meta_mirrorlist,
+ "config", &contents,
+ cancellable, error))
goto out;
ret_keyfile = g_key_file_new ();
ot_transfer_out_value (out_keyfile, &ret_keyfile);
out:
g_clear_pointer (&ret_keyfile, (GDestroyNotify) g_key_file_unref);
- g_clear_pointer (&target_uri, (GDestroyNotify) soup_uri_free);
return ret;
}
g_autoptr(GBytes) delta_superblock_data = NULL;
g_autoptr(GBytes) delta_meta_data = NULL;
g_autoptr(GVariant) delta_superblock = NULL;
- SoupURI *target_uri = NULL;
-
- target_uri = suburi_new (pull_data->base_content_uri, delta_name, NULL);
-
- if (!_ostree_fetcher_request_uri_to_membuf (pull_data->fetcher, target_uri,
- FALSE, TRUE,
- &delta_superblock_data,
- OSTREE_MAX_METADATA_SIZE,
- pull_data->cancellable, error))
+
+ if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
+ pull_data->content_mirrorlist,
+ delta_name, FALSE, TRUE,
+ &delta_superblock_data,
+ OSTREE_MAX_METADATA_SIZE,
+ pull_data->cancellable, error))
goto out;
-
+
if (delta_superblock_data)
{
{
if (out_delta_superblock)
*out_delta_superblock = g_steal_pointer (&ret_delta_superblock);
out:
- g_clear_pointer (&target_uri, (GDestroyNotify) soup_uri_free);
return ret;
}
const guchar *csum;
g_autoptr(GVariant) header = NULL;
gboolean have_all = FALSE;
- SoupURI *target_uri = NULL;
g_autofree char *deltapart_path = NULL;
FetchStaticDeltaData *fetch_data;
g_autoptr(GVariant) csum_v = NULL;
NULL, &inline_delta_part,
cancellable, error))
goto out;
-
+
_ostree_static_delta_part_execute_async (pull_data->repo,
fetch_data->objects,
inline_delta_part,
}
else
{
- target_uri = suburi_new (pull_data->base_content_uri, deltapart_path, NULL);
- _ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, target_uri, size,
- OSTREE_FETCHER_DEFAULT_PRIORITY,
- pull_data->cancellable,
- static_deltapart_fetch_on_complete,
- fetch_data);
+ _ostree_fetcher_mirrored_request_with_partial_async (pull_data->fetcher,
+ pull_data->content_mirrorlist,
+ deltapart_path, size,
+ OSTREE_FETCHER_DEFAULT_PRIORITY,
+ pull_data->cancellable,
+ static_deltapart_fetch_on_complete,
+ fetch_data);
pull_data->n_outstanding_deltapart_fetches++;
- soup_uri_free (target_uri);
}
}
static gboolean
_ostree_preload_metadata_file (OstreeRepo *self,
OstreeFetcher *fetcher,
- SoupURI *base_uri,
+ GPtrArray *mirrorlist,
const char *filename,
gboolean is_metalink,
GBytes **out_bytes,
glnx_unref_object OstreeMetalink *metalink = NULL;
GError *local_error = NULL;
+ /* the metalink uri is buried in the mirrorlist as the first (and only)
+ * element */
metalink = _ostree_metalink_new (fetcher, filename,
OSTREE_MAX_METADATA_SIZE,
- base_uri);
+ mirrorlist->pdata[0]);
_ostree_metalink_request_sync (metalink, NULL, out_bytes,
cancellable, &local_error);
}
else
{
- SoupURI *uri;
- const char *base_path;
- g_autofree char *path = NULL;
-
- base_path = soup_uri_get_path (base_uri);
- path = g_build_filename (base_path, filename, NULL);
- uri = soup_uri_new_with_base (base_uri, path);
-
- ret = _ostree_fetcher_request_uri_to_membuf (fetcher, uri,
- FALSE, TRUE,
- out_bytes,
- OSTREE_MAX_METADATA_SIZE,
- cancellable, error);
- soup_uri_free (uri);
+ ret = _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist,
+ filename, FALSE, TRUE,
+ out_bytes,
+ OSTREE_MAX_METADATA_SIZE,
+ cancellable, error);
if (!ret)
goto out;
return ret;
}
+static gboolean
+fetch_mirrorlist (OstreeFetcher *fetcher,
+ const char *mirrorlist_url,
+ GPtrArray **out_mirrorlist,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ char **lines = NULL;
+ g_autofree char *contents = NULL;
+ SoupURI *mirrorlist = NULL;
+ g_autoptr(GPtrArray) ret_mirrorlist =
+ g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
+
+ mirrorlist = soup_uri_new (mirrorlist_url);
+ if (mirrorlist == NULL)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Failed to parse mirrorlist URL '%s'", mirrorlist_url);
+ goto out;
+ }
+
+ if (!fetch_uri_contents_utf8_sync (fetcher, mirrorlist, &contents,
+ cancellable, error))
+ {
+ g_prefix_error (error, "While fetching mirrorlist '%s': ",
+ mirrorlist_url);
+ goto out;
+ }
+
+ /* go through each mirror in mirrorlist and do a quick sanity check that it
+ * works so that we don't waste the fetcher's time when it goes through them
+ * */
+ lines = g_strsplit (contents, "\n", -1);
+ g_debug ("Scanning mirrorlist from '%s'", mirrorlist_url);
+ for (char **iter = lines; iter && *iter; iter++)
+ {
+ const char *mirror_uri_str = *iter;
+ SoupURI *mirror_uri = NULL;
+
+ /* let's be nice and support empty lines and comments */
+ if (*mirror_uri_str == '\0' || *mirror_uri_str == '#')
+ continue;
+
+ mirror_uri = soup_uri_new (mirror_uri_str);
+ if (mirror_uri == NULL)
+ {
+ g_debug ("Can't parse mirrorlist line '%s'", mirror_uri_str);
+ continue;
+ }
+ else if ((strcmp (soup_uri_get_scheme (mirror_uri), "http") != 0) &&
+ (strcmp (soup_uri_get_scheme (mirror_uri), "https") != 0))
+ {
+ /* let's not support mirrorlists that contain non-http based URIs for
+ * now (e.g. local URIs) -- we need to think about if and how we want
+ * to support this since we set up things differently depending on
+ * whether we're pulling locally or not */
+ g_debug ("Ignoring non-http/s mirrorlist entry '%s'", mirror_uri_str);
+ soup_uri_free (mirror_uri);
+ continue;
+ }
+
+ /* We keep sanity checking until we hit a working mirror; there's no need
+ * to waste resources checking the remaining ones. At the same time,
+ * guaranteeing that the first mirror in the list works saves the fetcher
+ * time from always iterating through a few bad first mirrors. */
+ if (ret_mirrorlist->len == 0)
+ {
+ GError *local_error = NULL;
+ g_autofree char *config_uri_str = g_build_filename (mirror_uri_str,
+ "config", NULL);
+ SoupURI *config_uri = soup_uri_new (config_uri_str);
+
+ if (fetch_uri_contents_utf8_sync (fetcher, config_uri, NULL,
+ cancellable, &local_error))
+ g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri));
+ else
+ {
+ g_debug ("Failed to fetch config from mirror '%s': %s",
+ mirror_uri_str, local_error->message);
+ g_clear_error (&local_error);
+ }
+
+ soup_uri_free (config_uri);
+ }
+ else
+ {
+ g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri));
+ }
+
+ if (mirror_uri != NULL)
+ soup_uri_free (mirror_uri);
+ }
+
+ if (ret_mirrorlist->len == 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "No valid mirrors were found in mirrorlist '%s'",
+ mirrorlist_url);
+ goto out;
+ }
+
+ *out_mirrorlist = g_steal_pointer (&ret_mirrorlist);
+ ret = TRUE;
+
+out:
+ if (mirrorlist != NULL)
+ soup_uri_free (mirrorlist);
+ return ret;
+}
+
static gboolean
repo_remote_fetch_summary (OstreeRepo *self,
const char *name,
glnx_unref_object OstreeFetcher *fetcher = NULL;
g_autoptr(GMainContext) mainctx = NULL;
gboolean ret = FALSE;
- SoupURI *base_uri = NULL;
gboolean from_cache = FALSE;
g_autofree char *url_override = NULL;
+ g_autoptr(GPtrArray) mirrorlist = NULL;
if (options)
(void) g_variant_lookup (options, "override-url", "&s", &url_override);
else if (!ostree_repo_remote_get_url (self, name, &url_string, error))
goto out;
- base_uri = soup_uri_new (url_string);
- if (base_uri == NULL)
+ if (metalink_url_string == NULL &&
+ g_str_has_prefix (url_string, "mirrorlist="))
{
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Invalid URL '%s'", url_string);
- goto out;
+ if (!fetch_mirrorlist (fetcher, url_string + strlen ("mirrorlist="),
+ &mirrorlist, cancellable, error))
+ goto out;
+ }
+ else
+ {
+ SoupURI *uri = soup_uri_new (url_string);
+
+ if (uri == NULL)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Failed to parse url '%s'", url_string);
+ goto out;
+ }
+
+ mirrorlist =
+ g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
+ g_ptr_array_add (mirrorlist, uri /* transfer ownership */ );
}
}
if (!_ostree_preload_metadata_file (self,
fetcher,
- base_uri,
+ mirrorlist,
"summary.sig",
metalink_url_string ? TRUE : FALSE,
out_signatures,
{
if (!_ostree_preload_metadata_file (self,
fetcher,
- base_uri,
+ mirrorlist,
"summary",
metalink_url_string ? TRUE : FALSE,
out_summary,
out:
if (mainctx)
g_main_context_pop_thread_default (mainctx);
- if (base_uri != NULL)
- soup_uri_free (base_uri);
return ret;
}
gboolean opt_gpg_verify_set = FALSE;
gboolean opt_gpg_verify_summary_set = FALSE;
const char *url_override = NULL;
+ g_autofree char *base_meta_url = NULL;
+ g_autofree char *base_content_url = NULL;
if (options)
{
else if (!ostree_repo_remote_get_url (self, remote_name_or_baseurl, &baseurl, error))
goto out;
- pull_data->base_uri = soup_uri_new (baseurl);
-
- if (!pull_data->base_uri)
+ if (g_str_has_prefix (baseurl, "mirrorlist="))
{
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Failed to parse url '%s'", baseurl);
- goto out;
+ if (!fetch_mirrorlist (pull_data->fetcher,
+ baseurl + strlen ("mirrorlist="),
+ &pull_data->meta_mirrorlist,
+ cancellable, error))
+ goto out;
+ }
+ else
+ {
+ SoupURI *baseuri = soup_uri_new (baseurl);
+
+ if (baseuri == NULL)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Failed to parse url '%s'", baseurl);
+ goto out;
+ }
+
+ pull_data->meta_mirrorlist =
+ g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
+ g_ptr_array_add (pull_data->meta_mirrorlist, baseuri /* transfer */);
}
}
else
error))
goto out;
+ /* XXX: would be interesting to implement metalink as another source of
+ * mirrors here since we use it as such anyway (rather than the "usual"
+ * use case of metalink, which is only for a single target filename) */
{
+ /* reuse target_uri and take ownership */
g_autofree char *repo_base = g_path_get_dirname (soup_uri_get_path (target_uri));
- pull_data->base_uri = soup_uri_copy (target_uri);
- soup_uri_set_path (pull_data->base_uri, repo_base);
+ soup_uri_set_path (target_uri, repo_base);
+ pull_data->meta_mirrorlist =
+ g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
+ g_ptr_array_add (pull_data->meta_mirrorlist, target_uri);
}
pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT,
goto out;
if (contenturl == NULL)
- pull_data->base_content_uri = soup_uri_copy (pull_data->base_uri);
+ /* this is a bit hacky but greatly simplifies coding elsewhere; we take
+ * care in the out path to not double free if they're the same list */
+ pull_data->content_mirrorlist = pull_data->meta_mirrorlist;
else
{
- pull_data->base_content_uri = soup_uri_new (contenturl);
- if (!pull_data->base_content_uri)
+ if (g_str_has_prefix (contenturl, "mirrorlist="))
{
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Failed to parse contenturl '%s'", contenturl);
- goto out;
+ if (!fetch_mirrorlist (pull_data->fetcher,
+ contenturl + strlen ("mirrorlist="),
+ &pull_data->content_mirrorlist,
+ cancellable, error))
+ goto out;
+ }
+ else
+ {
+ SoupURI *contenturi = soup_uri_new (contenturl);
+
+ if (contenturi == NULL)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Failed to parse contenturl '%s'", contenturl);
+ goto out;
+ }
+
+ pull_data->content_mirrorlist =
+ g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
+ g_ptr_array_add (pull_data->content_mirrorlist,
+ contenturi /* transfer */);
}
}
}
&configured_branches, error))
goto out;
- if (strcmp (soup_uri_get_scheme (pull_data->base_uri), "file") == 0)
+ /* NB: we don't support local mirrors in mirrorlists, so if this passes, it
+ * means that we're not using mirrorlists (see also fetch_mirrorlist()) */
+ if (strcmp (soup_uri_get_scheme (pull_data->meta_mirrorlist->pdata[0]), "file") == 0)
{
- g_autoptr(GFile) remote_repo_path = g_file_new_for_path (soup_uri_get_path (pull_data->base_uri));
+ g_autoptr(GFile) remote_repo_path =
+ g_file_new_for_path (soup_uri_get_path (pull_data->meta_mirrorlist->pdata[0]));
pull_data->remote_repo_local = ostree_repo_new (remote_repo_path);
if (!ostree_repo_open (pull_data->remote_repo_local, cancellable, error))
goto out;
pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
{
- SoupURI *uri = NULL;
g_autoptr(GBytes) bytes_sig = NULL;
g_autofree char *ret_contents = NULL;
gsize i, n;
if (!pull_data->summary_data_sig)
{
- uri = suburi_new (pull_data->base_uri, "summary.sig", NULL);
- if (!_ostree_fetcher_request_uri_to_membuf (pull_data->fetcher, uri,
- FALSE, TRUE, &bytes_sig,
- OSTREE_MAX_METADATA_SIZE,
- cancellable, error))
+ if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
+ pull_data->meta_mirrorlist,
+ "summary.sig", FALSE, TRUE,
+ &bytes_sig,
+ OSTREE_MAX_METADATA_SIZE,
+ cancellable, error))
goto out;
- soup_uri_free (uri);
}
if (bytes_sig &&
if (!pull_data->summary && !bytes_summary)
{
- uri = suburi_new (pull_data->base_uri, "summary", NULL);
- if (!_ostree_fetcher_request_uri_to_membuf (pull_data->fetcher, uri,
- FALSE, TRUE, &bytes_summary,
- OSTREE_MAX_METADATA_SIZE,
- cancellable, error))
+ if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
+ pull_data->meta_mirrorlist,
+ "summary", FALSE, TRUE,
+ &bytes_summary,
+ OSTREE_MAX_METADATA_SIZE,
+ cancellable, error))
goto out;
- soup_uri_free (uri);
}
if (!bytes_summary && pull_data->gpg_verify_summary)
g_clear_object (&pull_data->cancellable);
g_clear_object (&pull_data->remote_repo_local);
g_free (pull_data->remote_name);
- if (pull_data->base_uri)
- soup_uri_free (pull_data->base_uri);
- if (pull_data->base_content_uri)
- soup_uri_free (pull_data->base_content_uri);
+ if (pull_data->content_mirrorlist != pull_data->meta_mirrorlist)
+ g_clear_pointer (&pull_data->content_mirrorlist, (GDestroyNotify) g_ptr_array_unref);
+ /* we clear this *after* clearing content_mirrorlist to avoid unref'ing twice */
+ g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->summary_data, (GDestroyNotify) g_bytes_unref);
g_clear_pointer (&pull_data->summary_data_sig, (GDestroyNotify) g_bytes_unref);
g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref);
projects / ostree.git / commitdiff